#第八章 自然语言的处理

from utils.commonUtil import get_bs4_obj
from urllib.request import urlopen
import re
import string
import operator

def cleanInput(inputStr):
    inputStr = re.sub("\n+| +"," ",inputStr)
    inputStr = re.sub("\[[0-9]*\]","",inputStr) #去掉字符串中的 [0],[1] 这样的数据
    inputStr = bytes(inputStr,"utf-8")
    inputStr = inputStr.decode("ascii","ignore")
    inputStr = inputStr.split(" ")
    cleanInputData = []
    for i in inputStr:
        i = i.strip(string.punctuation) #去除两端的特殊符号
        if len(i)>1 or i.lower()=="a" or i.lower()=='i':
            cleanInputData.append(i)
    
    return cleanInputData

def ngrams(inputStr,n):
    inputStr = cleanInput(inputStr)
    num = len(inputStr)
    output = {}
    for i in range(num+n-1): #分析词语出现的次数
        ngramsTemp = " ".join(inputStr[i:i+n])
        if ngramsTemp not in output:
            output[ngramsTemp]=0
        output[ngramsTemp] += 1
    return output


with open("chp8_content.txt",'r') as f:
    c = f.read()
    print(type(c))


url = "http://pythonscraping.com/files/inaugurationSpeech.txt"
#content = str(urlopen(url).read(),'utf-8') # 读取文档 并转化成utf8编码的字符
ng = ngrams(c,2)
ngSort = sorted(ng.items(),key=operator.itemgetter(1),reverse=True)
print(ngSort)






